from cmath import pi
from genericpath import exists
from importlib.resources import path
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation
import pandas as pd
#import os
import jieba
import numpy as np
import pickle
import os
def stopwordslist():
    data=pd.read_csv("./stopwords.csv")
    return set(data["stopword"])

def chinese_word_cut(mytext):
    return jieba.cut_for_search(mytext)
def print_top_words(model, feature_names, n_top_words):
    for topic_idx, topic in enumerate(model.components_):
        print("Topic #%d:" % topic_idx)
        print(" ".join([feature_names[i]
                        for i in topic.argsort()[:-n_top_words - 1:-1]]))
        print()



df = pd.read_csv("./output.csv",encoding="gbk")
df=df.fillna('')



stopwords = stopwordslist()
dups=[]
for index,row in df.iterrows():
    cutwords=set(chinese_word_cut(row["content"]))
    dup=cutwords-stopwords
    dups.append(" ".join(list(dup)))
df["content_cutted"]=dups

n_features = 1000
tf_vectorizer = TfidfVectorizer()

tf = tf_vectorizer.fit_transform(dups)


n_topics = 5
lda = LatentDirichletAllocation(n_components=n_topics, max_iter=100,
                                learning_method='online',
                                learning_offset=10.,
                                learning_decay=0.8,
                                batch_size=256,
                                evaluate_every=50,
                                random_state=0)
if not os.path.exists("lda.pkl"):
    lda.fit(tf)
    pickle.dump(lda, open("lda.pkl", "wb"))
else:
    lda = pickle.load(open("lda.pkl", "rb"))
    
n_top_words = 20
tf_feature_names = tf_vectorizer.get_feature_names()

print_top_words(lda, tf_feature_names, n_top_words)
X_new = lda.transform(tf)

topic_idx = [x_i.argmax() for x_i in X_new]
from collections import Counter
cnt = Counter(topic_idx)
print(cnt)
df["topic_idx"] = topic_idx
df.to_csv("out_lda.csv")

import pyLDAvis
import pyLDAvis.sklearn
data = pyLDAvis.sklearn.prepare(lda, tf, tf_vectorizer)
pyLDAvis.show(data)
